import requests
from lxml import etree

counter = 0
# 一级分类
url = "https://www.xinpianchang.com/discover/article"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
response = requests.get(url, headers=headers)
result = response.content.decode()
print(result,response.status_code)
root = etree.HTML(result)
category_list = root.xpath(
    "//a[@class='rounded cursor-pointer mr-2.5 mb-2.5 transition-colors order-2 bg-gray-100 text-gray-500 hover:text-gray-800 hover:bg-gray-200']")
print(len(category_list))
for category in category_list:
    category_name = "".join(category.xpath(".//text()"))
    category_url = "https://www.xinpianchang.com" + category.xpath("./@href")[0]
    print(category_name, category_url)

    # 二级分类
    response = requests.get(category_url, headers=headers)
    result = response.content.decode()
    root = etree.HTML(result)
    sub_category_list = root.xpath(
        "//a[@class='rounded cursor-pointer mr-2.5 mb-2.5 transition-colors bg-gray-100 text-gray-500 hover:text-gray-800 hover:bg-gray-200']")
    for sub_category in sub_category_list:
        counter += 1
        sub_category_name = "".join(sub_category.xpath(".//text()"))
        sub_category_url = "https://www.xinpianchang.com" + sub_category.xpath("./@href")[0]
        print(counter, category_name, sub_category_name, sub_category_url)


